﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Emoney.Spider.Basic;
using System.Text.RegularExpressions;
using System.Data;

/******************************************************     
FileName: CmsGather.cs
Copyright  (c)  2010 上海益盟软件技术有限公司
Writer: xiaoll
create Date: 2010-10-18
Rewriter: 
Rewrite Date: 2010-10-18
Impact: 
Main Content 
******************************************************/

namespace Emoney.Cms.Spider
{
    public class CmsGather
    {
        LogFile log = new LogFile();

        /// <summary>
        /// 采集凤凰网行业报告
        /// </summary>
        /// <param name="p">采集多少页</param>
        public void FromIFengHYBG(int p,out int num)
        {
            num = 0;
            int _cid = 1; //行业报告

            try
            {
                Console.WriteLine("开始采集IFeng行业报告列表");
                log.WriteLog("开始采集IFeng行业报告列表");

                if (p < 1)
                    p = 20;

                StringBuilder sql = new StringBuilder();
                CmsAccess cAcc = new CmsAccess();
                List<CmsEnty> ListAll = (List<CmsEnty>)cAcc.ListAcc_NewsByCID(_cid);
                for (int n = 0; n < p; n++)
                {
                    string _url = "http://app.finance.ifeng.com/info/column1.php?sub_name=%E8%A1%8C%E4%B8%9A%E6%8A%A5%E5%91%8A&page=" + (n + 1) + "";
                    string _Content = GatherHtml.GetPageByUrl(_url, "utf-8");
                    Match mcSubCon = Regex.Match(_Content, "(?<=class=\"contentL\")[\\s\\S]*?(?=class=\"box02\">)", RegexOptions.IgnoreCase);
                    string _curl = "", _title = "";
                    DateTime? _ptime;
                    if (!string.IsNullOrEmpty(mcSubCon.Value))
                    {
                        MatchCollection mcItem = Regex.Matches(mcSubCon.Value, "<li[\\S\\s]*?</li>", RegexOptions.IgnoreCase);
                        for (int i = 0; i < mcItem.Count; i++)
                        {
                            try
                            {
                                _curl = "";
                                _title = "";
                                _ptime = null;
                                Match mcTit = Regex.Match(mcItem[i].Value, "(?<=target='_blank'>)[\\s\\S]*?(?=</a>)", RegexOptions.IgnoreCase);
                                Match mcUrl = Regex.Match(mcItem[i].Value, "(?<=href=')[\\s\\S]*?(?=')", RegexOptions.IgnoreCase);
                                Match mcTime = Regex.Match(mcItem[i].Value, "(?<=class=\'time01\'>)[\\s\\S]*?(?=</span>)", RegexOptions.IgnoreCase);
                                HtmlTags ht = new HtmlTags();
                                ht.t_html = true;

                                if (mcUrl.Length > 0)
                                    _curl = DelHtmlTags.DelHtml(mcUrl.Value, ht);
                                if (mcTit.Length > 0)
                                    _title = DelHtmlTags.DelHtml(mcTit.Value.Trim(), ht);
                                if (mcTime.Length > 0)
                                    _ptime = OTO.ToDate(DelHtmlTags.DelHtml(mcTime.Value, ht).Replace("(", "").Replace(")", ""), DateTime.Now);

                                if (!string.IsNullOrEmpty(_curl) && !string.IsNullOrEmpty(_title))
                                {
                                    CmsEnty et = new CmsEnty();
                                    et.LINK = _curl;
                                    et.TITLE = _title;
                                    et.PTIME = _ptime;
                                    et.CTIME = DateTime.Now;
                                    et.FLAG = -1;
                                    et.CID = _cid;
                                    if (ListAll.Count > 0)
                                    {
                                        List<CmsEnty> MidList = (List<CmsEnty>)ListAll.Where(o => o.CID.Equals(_cid) && o.LINK.Equals(_curl)).ToList();
                                        if (MidList == null || MidList.Count < 1)
                                        {
                                            int m = 0;
                                            m = cAcc.InsertNews(et);
                                            num++;
                                            Console.WriteLine("IFeng行业报告采集成功" + m.ToString() + "条，链接：" + _curl + "");
                                            log.WriteLog("IFeng行业报告采集成功" + m.ToString() + "条，链接：" + _curl + "");
                                        }
                                    }
                                    else
                                    {
                                        int m = 0;
                                        m = cAcc.InsertNews(et);
                                        num++;
                                        Console.WriteLine("IFeng行业报告采集列表成功" + m.ToString() + "条，链接：" + _curl + "");
                                        log.WriteLog("IFeng行业报告采集列表成功" + m.ToString() + "条，链接：" + _curl + "");
                                    }
                                }
                            }
                            catch
                            {
                                Console.WriteLine("IFeng行业报告链接" + _curl + "出错");
                                log.WriteLog("IFeng行业报告链接" + _curl + "出错");
                            }
                        }
                    }
                }
            }
            catch
            {
                log.WriteLog("IFeng行业报告预载列表出错或执行sql出错");
            }
            finally
            {
                string key = string.Format(CacheKey.News_CACHE_KEY, "DB", _cid.ToString());
                Cache.MoveCache(key);
            }
        }

        /// <summary>
        /// 采集更新凤凰网行业报告文章内容
        /// </summary>
        public void UpdateIFengHYBG()
        {
            int _cid = 1; //行业报告
            try
            {
                CmsAccess cAcc = new CmsAccess();
                List<CmsEnty> ListAll = (List<CmsEnty>)cAcc.ListAcc_NewsByCID(_cid).Where(o => o.FLAG.Equals(-1)).ToList();
                for (int i = 0; i < ListAll.Count; i++)
                {
                    string _Content = GatherHtml.GetPageByUrl(ListAll[i].LINK, "utf-8");
                    if (!string.IsNullOrEmpty(_Content))
                    {
                        HtmlTags tags = new HtmlTags();
                        CmsEnty et = new CmsEnty();
                        Match mcSource = Regex.Match(_Content, "(?<=id=\"source_place\">)[\\s\\S]*?(?=</span>)", RegexOptions.IgnoreCase);
                        if (mcSource.Length > 0)
                        {
                            tags.t_html = true;
                            et.SOURCE = DelHtmlTags.DelHtml(mcSource.Value, tags);
                        }

                        Match mcContent = Regex.Match(_Content, "(?<=id=artical_real>)[\\s\\S]*?(?=<p>更多数据,请访问)", RegexOptions.IgnoreCase);
                        if (mcContent.Length > 0)
                        {
                            tags = new HtmlTags();
                            tags = tags.ThrDelHtml();
                            tags.t_img = true;
                            et.CONTENT = DelHtmlTags.DelHtml(mcContent.Value, tags);
                        }

                        //<div id="inner_hq" class="insertBlock"></div>
                        //删除采集内容中插入行情内容块
                        et.CONTENT = Regex.Replace(et.CONTENT, "<div\\s*id=\"inner_hq\"[^>]+>[\\S\\s]*?</div>", "", RegexOptions.IgnoreCase);
                        //<p>?</p>
                        //异常标签
                        et.CONTENT = et.CONTENT.Replace("<p>?</p>", "");

                        if (!string.IsNullOrEmpty(et.CONTENT))
                        {
                            int m = 0;
                            et.ID = (int)ListAll[i].ID;
                            et.FLAG = 0;
                            m = cAcc.UpdateNews(et);
                            Console.WriteLine("IFeng行业报告采集内容更新成功" + m.ToString() + "条，链接：" + ListAll[i].LINK + "");
                            log.WriteLog("IFeng行业报告采集内容更新成功" + m.ToString() + "条，链接：" + ListAll[i].LINK + "");
                        }
                        else  //取不到内容的，flag不变，更新次+1
                        {
                            int m = 0;
                            et.ID = (int)ListAll[i].ID;
                            et.FLAG = -1;
                            m = cAcc.UpdateNews(et);
                        }
                    }
                }
            }
            catch
            {
                Console.WriteLine("更新内容操作：IFeng行业报告更新内容时预载列表出错或执行sql出错");
                log.WriteLog("更新内容操作：IFeng行业报告更新内容时预载列表出错或执行sql出错");
            }
            finally
            {
                string key = string.Format(CacheKey.News_CACHE_KEY, "DB", _cid.ToString());
                Cache.MoveCache(key);
            }
        }


        /// <summary>
        /// 采集证星行业报告
        /// </summary>
        /// <param name="p">采集多少页</param>
        public void FromStockStarHYBG(int p,out int num)
        {
            num = 0;
            int _cid = 1; //行业报告
            try
            {
                Console.WriteLine("开始采集StockStar行业报告列表");
                log.WriteLog("开始采集StockStar行业报告列表");

                if (p < 1)
                    p = 20;

                StringBuilder sql = new StringBuilder();
                CmsAccess cAcc = new CmsAccess();
                List<CmsEnty> ListAll = (List<CmsEnty>)cAcc.ListAcc_NewsByCID(_cid);
                for (int n = 0; n < p; n++)
                {
                    string _url = "http://stock.stockstar.com/list/3489_" + (n + 1) + ".shtml";
                    string _Content = GatherHtml.GetPageByUrl(_url, "gb2312");
                    Match mcSubCon = Regex.Match(_Content, "(?<=class=\"listnews\">)[\\s\\S]*?(?=class=\"pageControl\">)", RegexOptions.IgnoreCase);
                    string _curl = "", _title = "";
                    DateTime? _ptime;
                    if (!string.IsNullOrEmpty(mcSubCon.Value))
                    {
                        MatchCollection mcItem = Regex.Matches(mcSubCon.Value, "<li[\\S\\s]*?</li>", RegexOptions.IgnoreCase);
                        for (int i = 0; i < mcItem.Count; i++)
                        {
                            try
                            {
                                _curl = "";
                                _title = "";
                                _ptime = null;
                                Match mcTit = Regex.Match(mcItem[i].Value, "<a[\\S\\s]*?</a>", RegexOptions.IgnoreCase);
                                Match mcUrl = Regex.Match(mcItem[i].Value, "(?<=href=\")[\\s\\S]*?(?=\")", RegexOptions.IgnoreCase);
                                Match mcTime = Regex.Match(mcItem[i].Value, "(?<=<span>)[\\s\\S]*?(?=</span>)", RegexOptions.IgnoreCase);
                                HtmlTags ht = new HtmlTags();
                                ht.t_html = true;

                                if (mcUrl.Length > 0)
                                    _curl = DelHtmlTags.DelHtml(mcUrl.Value, ht);
                                if (mcTit.Length > 0)
                                    _title = DelHtmlTags.DelHtml(mcTit.Value.Trim(), ht);
                                if (mcTime.Length > 0)
                                    _ptime = OTO.ToDate(DelHtmlTags.DelHtml(mcTime.Value, ht), DateTime.Now);

                                if (!string.IsNullOrEmpty(_curl) && !string.IsNullOrEmpty(_title))
                                {
                                    CmsEnty et = new CmsEnty();
                                    et.LINK = _curl;
                                    et.TITLE = _title;
                                    et.PTIME = _ptime;
                                    et.CTIME = DateTime.Now;
                                    et.FLAG = -1;
                                    et.CID = _cid;
                                    if (ListAll.Count > 0)
                                    {
                                        List<CmsEnty> MidList = (List<CmsEnty>)ListAll.Where(o => o.CID.Equals(_cid) && o.LINK.Equals(_curl)).ToList();
                                        if (MidList == null || MidList.Count < 1)
                                        {
                                            int m = 0;
                                            m = cAcc.InsertNews(et);
                                            num++;
                                            Console.WriteLine("StockStar行业报告采集成功" + m.ToString() + "条，链接：" + _curl + "");
                                            log.WriteLog("StockStar行业报告采集成功" + m.ToString() + "条，链接：" + _curl + "");
                                        }
                                    }
                                    else
                                    {
                                        int m = 0;
                                        m = cAcc.InsertNews(et);
                                        num++;
                                        Console.WriteLine("StockStar行业报告采集列表成功" + m.ToString() + "条，链接：" + _curl + "");
                                        log.WriteLog("StockStar行业报告采集列表成功" + m.ToString() + "条，链接：" + _curl + "");
                                    }
                                }
                            }
                            catch
                            {
                                Console.WriteLine("StockStar行业报告链接" + _curl + "出错");
                                log.WriteLog("StockStar行业报告链接" + _curl + "出错");
                            }
                        }
                    }
                }
            }
            catch
            {
                log.WriteLog("IFeng行业报告预载列表出错或执行sql出错");
            }
            finally
            {
                string key = string.Format(CacheKey.News_CACHE_KEY, "DB", _cid.ToString());
                Cache.MoveCache(key);
            }
        }

        /// <summary>
        /// 采集更新证星行业报告文章内容
        /// </summary>
        public void UpdateStockStarHYBG()
        {
            int _cid = 1; //行业报告
            try
            {
                CmsAccess cAcc = new CmsAccess();
                List<CmsEnty> ListAll = (List<CmsEnty>)cAcc.ListAcc_NewsByCID(_cid).Where(o => o.FLAG.Equals(-1)).ToList();
                for (int i = 0; i < ListAll.Count; i++)
                {
                    string _Content = GatherHtml.GetPageByUrl(ListAll[i].LINK, "gb2312");
                    if (!string.IsNullOrEmpty(_Content))
                    {
                        HtmlTags tags = new HtmlTags();
                        CmsEnty et = new CmsEnty();
                        Match mcSource = Regex.Match(_Content, "(?<=id=\"con1\")[\\s\\S]*?(?=</div>)", RegexOptions.IgnoreCase);
                        if (mcSource.Length > 0)
                        {
                            MatchCollection mcSource2 = Regex.Matches(mcSource.Value, "<a[\\S\\s]*?</a>", RegexOptions.IgnoreCase);
                            if (mcSource2.Count > 0)
                            {
                                tags.t_html = true;
                                et.SOURCE = DelHtmlTags.DelHtml(mcSource2[mcSource2.Count - 1].Value, tags);
                            }
                        }

                        Match mcContent = Regex.Match(_Content, "(?<=<div\\s*id=\"NewsContentText\"[^>]+>)[\\s\\S]*?(?=<div class=\"con5\">)", RegexOptions.IgnoreCase);
                        if (mcContent.Length > 0)
                        {
                            tags = new HtmlTags();
                            tags = tags.ThrDelHtml();
                            tags.t_img = true;
                            et.CONTENT = DelHtmlTags.DelHtml(mcContent.Value, tags);
                            //进一步处理多余标签</div>
                            //</div>\r\n        "
                            et.CONTENT = et.CONTENT.Substring(0, et.CONTENT.LastIndexOf("</div>"));
                            et.CONTENT = et.CONTENT.Replace("相关附件", "");
                        }

                        if (!string.IsNullOrEmpty(et.CONTENT))
                        {
                            int m = 0;
                            et.ID = (int)ListAll[i].ID;
                            et.FLAG = 0;
                            m = cAcc.UpdateNews(et);
                            Console.WriteLine("证星行业报告采集内容更新成功" + m.ToString() + "条，链接：" + ListAll[i].LINK + "");
                            log.WriteLog("证星行业报告采集内容更新成功" + m.ToString() + "条，链接：" + ListAll[i].LINK + "");
                        }
                        else  //取不到内容的，flag不变，更新次+1
                        {
                            int m = 0;
                            et.ID = (int)ListAll[i].ID;
                            et.FLAG = -1;
                            m = cAcc.UpdateNews(et);
                        }
                    }
                }
            }
            catch
            {
                Console.WriteLine("更新内容操作：证星行业报告更新内容时预载列表出错或执行sql出错");
                log.WriteLog("更新内容操作：证星行业报告更新内容时预载列表出错或执行sql出错");
            }
            finally
            {
                string key = string.Format(CacheKey.News_CACHE_KEY, "DB", _cid.ToString());
                Cache.MoveCache(key);
            }
        }
    }
}
